import argparse
import json
import os
import sys
sys.path.append('..')

from pycg.pycg import CallGraphGenerator
from pycg import formats
from pycg.utils.constants import CALL_GRAPH_OP, KEY_ERR_OP

from callgraph_gen.readonly_neo4jDBOP import readonly_neo4jDBOP
from neo4jDBOP import neo4jDBOP
from import_graph.import_graph import ImportGraph

import astunparse
import requests
import wget
import tarfile
import datetime
import logging

from data_structure import FileSystemNode

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

logfile = "logs/log_for_nonexistentAPI_%s.txt" % datetime.date.today().strftime('%y%m%d')
fh = logging.FileHandler(logfile, mode='a')
fh.setLevel(logging.INFO)

ch = logging.StreamHandler()
ch.setLevel(logging.INFO)

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)

logger.addHandler(fh)
logger.addHandler(ch)

neo4j_conf = {
    "uri" : "neo4j://211.71.15.39:7685",
    "user" : "neo4j",
    "password" : "pysr"
}
# browser : 211.71.15.39:7472
# 211.71.15.39:7471

neo4j_conf_readonly = {
    "uri" : "neo4j://211.71.15.39:9687",
    "user" : "neo4j",
    "password" : "python"
}
# browser : 211.71.15.39:9474
# 211.71.15.39:9473

DBOP = neo4jDBOP(neo4j_conf=neo4j_conf)

def get_prefix(path, prefix):
    """ 获取该文件下所有可能的import路径 """
    """ path可以是目录的绝对路径或相对路径 
        prefix为以python第三方包名的开始的路径 
    """
    pyfiles = []
    valid_suffixes = ('.py')
    for file in os.listdir(path):
        file_path = os.path.join(path, file)
        dir_prefix = prefix + file
        if os.path.isdir(file_path):
            pyfiles.append(dir_prefix.replace('/', '.'))
            pyfiles.extend(get_prefix(file_path, dir_prefix))
        else:
            if file.endswith(valid_suffixes):
                file_name = file[:-3]
                file_prefix = prefix + file_name
                pyfiles.append(file_prefix.replace('/', '.'))
    return pyfiles

def clasify(pkg_name, prefix, pkg_json):
    """ 对pkg_json中的API进行分类 """
    inerAPI = []
    outerAPI = []
    for key, value in pkg_json.items():
        flag = True
        API_name = pkg_name + "." + key
        for pre in prefix:
            if API_name.startswith(pre):
                inerAPI.append(API_name)
                flag = False
                break
        if flag:
            outerAPI.append(key)

    result = {"prefix" : prefix, "inerAPI": inerAPI, "outerAPI" : outerAPI}
    f = open("requests_clasify.json", 'w')
    f.write(json.dumps(result))
    f.close()

    return inerAPI, outerAPI

class PkgAPIs(object):
    def __init__(self, pkg_name, version):
        # self.APIpairs_path = "../json/APIpairs/"
        self.callgraph_path = "/home/sdp/science/sr/PySC_API/callgraphs/"
        self.dir_name = "/home/sdp/science/sr/PySC_API/pkgs/" + pkg_name + "-" + version + "/" # ../pkgs/requests-2.28.2/ 安装包解压后的相对路径名
        self.pkg_name = pkg_name # requests
        self.version = version # 2.28.2
        self.dependencies = [] # [(name, version)] 所有依赖
        self.deptoanalyse = [] # 未分析的依赖
        self.pyfiles = []
        self.invalid_prefix = "<builtin>"
        self.API_dict = None
        self.API_pairs = {}
    
    def get_prefix(self, path, prefix):
        """ 获取该文件下所有可能的import路径 """
        """ path可以是目录的绝对路径或相对路径 
            prefix为以python第三方包名的开始的路径 
        """
        valid_suffixes = ('.py')
        for file in os.listdir(path):
            file_path = os.path.join(path, file)
            dir_prefix = prefix + "/" + file
            if os.path.isdir(file_path):
                self.myAPIs.append(dir_prefix.replace('/', '.'))
                self.get_prefix(file_path, dir_prefix)
            else:
                if file.endswith(valid_suffixes):
                    file_name = file[:-3]
                    file_prefix = prefix + "/" + file_name
                    self.myAPIs.append(file_prefix.replace('/', '.'))
                    self.pyfiles.append(os.path.abspath(file_path))
        
        file_name = self.pkg_name + "-" + self.version + ".json"
        f = open(file_name, 'w')
        f.write(json.dumps(self.myAPIs))
        f.close()

    def get_pyfiles(self, path):
        valid_suffixes = ('.py')
        for file in os.listdir(path):
            file_path = os.path.join(path, file)
            if os.path.isdir(file_path):
                self.get_pyfiles(file_path)
            else:
                if file.endswith(valid_suffixes):
                    self.pyfiles.append(os.path.abspath(file_path))
        # print(self.pyfiles)

    # def check_third_party_API(self, API_name):
    #     """ 检查某个API是否为第三方API """
    #     name = self.pkg_name + "." + API_name
    #     for prefix in self.myAPIs:
    #         if name.startswith(prefix):
    #             return False
    #     for prefix in self.invalid_prefix:
    #         if API_name.startswith(prefix):
    #             return False
    #     return True
    
    # def check_myAPI(self, API_name):
    #     """ 检查某个API是否为自定义的API """
    #     name = self.pkg_name + "." + API_name
    #     for prefix in self.myAPIs:
    #         if name.startswith(prefix):
    #             return True
    #     return False
    
    # def get_APIpairs(self):
    #     """ 处理callgraph中的结果，提取自定义API中的调用关系 """
    #     for key, value in self.API_dict.items():
    #         if self.check_myAPI(key):
    #             self.API_pairs[key] = []
    #             for called_API in value:
    #                 for prefix in self.invalid_prefix:
    #                     if not called_API.startswith(prefix):
    #                         self.API_pairs[key].append(called_API)
    #         # else:
    #         #     print(key)
    #     file_name = self.APIpairs_path + self.pkg_name + "-" + self.version + ".json"
    #     f = open(file_name, 'w')
    #     f.write(json.dumps(self.API_pairs))
    #     f.close()

    def get_dependencies(self):
        """ 从包级别依赖数据库中获取某个该python包的所有依赖包 """
        readonly_DBOP = readonly_neo4jDBOP(neo4j_conf_readonly)
        # readonly_DBOP = readonly_neo4jDBOP(neo4j_conf)
        dps_list = readonly_DBOP.get_dependencies(self.pkg_name, self.version)
        for item in dps_list:
            self.dependencies[item[0]] = [item[1], False]
        # print(self.dependencies)

    def callgraph_gen(self):
        """ 生成callgraph，以json形式返回 """
        # pycg --package /Users/mac/Desktop/PySC_API/callgraph_gen/pkgs/requests-2.28.2 $(find /Users/mac/Desktop/PySC_API/callgraph_gen/pkgs/requests-2.28.2 -type f -name "*.py") -o requests.json
        
        # 分析pkg：name-version 生成callgraph
        # flag = DBOP.checkPkgAnalysed(name=self.pkg_name, version=self.version)
        # if len(flag) == 0 or not flag[0]:

        cg = CallGraphGenerator(self.pyfiles, self.dir_name, -1, CALL_GRAPH_OP)
        cg.analyze()

        formatter = formats.Simple(cg)
        output = formatter.generate()

        self.API_dict = output

        json_file = self.callgraph_path + self.pkg_name + "-" + self.version + ".json"
        with open(json_file, "w+") as f:
            f.write(json.dumps(self.API_dict))

        DBOP.labelPkgAnalysed(self.pkg_name, self.version)

        return json_file
    
    def callgraph_store(self):
        for key, value in self.API_dict.items():
            if key.startswith(self.invalid_prefix): continue
            package_name = self.pkg_name + "-" + self.version
            if len(value) == 0: continue
            result = DBOP.searchAPI(package_name + "." + key) 
            if len(result) == 0: continue # 依赖的API
            # 检查API是否存在，package的所有依赖都已经处理过，期望结果是一定会找到相应API
            # 如果是package内部的API，路径名中缺少了当前包名，依赖的API是完整的，理论上一定可以找到

            caller = package_name + "." + key
            for called_API in value:
                if called_API.startswith(self.invalid_prefix): continue
                result = DBOP.searchAPI(package_name + ".")
                if len(result) == 0:
                    result = DBOP.searchAPI(called_API)
                    if len(result) == 0:
                        logger.INFO("%s is not exist, called by %s, %s." % (called_API, caller, self.pkg_name + "@" + self.version))
                        continue
                    callee = called_API
                else:
                    callee = package_name + "." + called_API
                DBOP.insertCallRelation(caller=caller, callee=callee)                     

            # result = DBOP.searchAPI(root_file=self.pkg_name, path=path)
            # print(result)
            # if self.check_myAPI(key):
            #     self.API_pairs[key] = []
            #     for called_API in value:
            #         for prefix in self.invalid_prefix:
            #             if not called_API.startswith(prefix):
            #                 self.API_pairs[key].append(called_API)

# if __name__ == "__main__":
    # readonly_neo4jDBOP = readonly_neo4jDBOP(neo4j_conf)
    # neo4jDBOP = readonly_neo4jDBOP(neo4j_conf)

    
    
    # result = pkgAPIs.get_dependencies()
    # dependencies = get_dependencies("requests", "2.28.2")
    # pkgAPIs.dependencies = dependencies
    # pkgAPIs = PkgAPIs("pkgs/urllib3-1.26.14", "urllib3", "1.26.14")

    # pkgAPIs.get_prefix(pkgAPIs.dir_name, pkgAPIs.pkg_name)
    # pkgAPIs.callgraph_gen()
    # pkgAPIs.get_APIpairs()

    # flag = DBOP.checkPkgDownloaded(name="requests", version="2.28.2")
    # print(flag)

    # pkgAPIs.download_dependencies("requests", "2.30.0")
    # path = os.path.abspath("pkgs/requests-2.30.0")
    # print(path)
    
    # pkgAPIs = PkgAPIs("requests", "2.28.2")
    # pkgAPIs.download_dependencies(pkgAPIs.pkg_name, pkgAPIs.version)
    # pkgAPIs.callgraph_gen()
    # pkgAPIs.callgraph_store()